﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using WebCommit.Common.Entitys;
using System.Threading;
using System.ComponentModel;
using WebCommit.Common.Enums;
using System.Net;
using System.IO;
using WebCommit.Common;
using NHibernate;
using System.Text.RegularExpressions;
using WebCommit.Common.Helpers;

namespace WebCommit.Crawler
{
	public class UrlCrawlWorker : Worker
	{
		public UrlCrawlWorker(List<WebResourceEntity> webList, object lockObj)
			: base(webList, lockObj)
		{ }

		private object lockObj = new object();
		public override List<WebResourceEntity> PreWork(List<WebResourceEntity> webList)
		{
			List<WebResourceEntity> list = new List<WebResourceEntity>();
			lock (WebListLockObj)
			{
				if (webList.Count(w => w.WebCrawlStatus == CrawlStatus.Pending) < 200)
				{

					WebResourceEntity web = webList.FirstOrDefault(w => (w.UrlCrawlStatus == CrawlStatus.Pending && w.WebCrawlStatus != CrawlStatus.Crawling));
					if (web != null)
					{
						web.UrlCrawlStatus = CrawlStatus.Crawling;
						web.WebCrawlStatus = CrawlStatus.Crawling;
						list.Add(web);
					}
				}
			}
			if (list.Count == 0)
			{
				Thread.Sleep(10);
			}
			return list;
		}

		public override List<WebResourceEntity> DoWork(List<WebResourceEntity> webList)
		{
			if (webList.Count < 1)
				return null;
			WebResourceEntity web = webList[0];
			List<WebResourceEntity> list = new List<WebResourceEntity>();
			HttpWebResponse resp = null;
			try
			{

				HttpWebRequest req = Utility.CreateWebRequest(web.Url);
				resp = (HttpWebResponse)req.GetResponse();
				if (resp.ContentType.ToLower().IndexOf("text/html") < 0)
				{
					lock (WebListLockObj)
					{
						web.UrlCrawlStatus = CrawlStatus.Cancel;
						web.WebCrawlStatus = CrawlStatus.Cancel;
					}

					Utility.Log("Cancel: ContentType= " + resp.ContentType, null);
					return webList;
				}
				string charSet = "";
				string html = Parser.ExtractHtml(resp, ref charSet);
				lock (WebListLockObj)
				{
					web.Title = Parser.ExtractTitle(html);
					web.CharacterSet = charSet;
				}

				string baseUri = Utility.GetBaseUri(web.Url);
				//string[] links = Parser.ExtractLinks(baseUri, html);

				string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
				MatchCollection matches = new Regex(strRef).Matches(html);

				foreach (Match match in matches)
				{
					strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
					try
					{
						if (Parser.IsGoodUri(strRef))
						{
							Utility.Normalize(baseUri, ref strRef);
							lock (WebListLockObj)
							{
								if (!IsUseful(strRef))
									continue;

								this.WebList.Add(CreateWebResource(strRef));
							}
						}
					}
					catch (Exception)
					{
					}
				}
				//foreach (string link in links)
				//{
				//    //if (WebList.Count(w => w.Url == link) < 1)
				//    //    list.Add(CreateWebResource(link));
				//    if(UrlHistory.Contains(link))
				//        continue;
				//    list.Add(CreateWebResource(link));
				//    UrlHistory.Add(link);
				//}
				lock (WebListLockObj)
				{
					web.UrlCrawlStatus = CrawlStatus.OK;
					web.WebCrawlStatus = CrawlStatus.OK;
					//this.WebList.AddRange(list);
				}
				return list;
			}
			catch (Exception ex)
			{
				lock (WebListLockObj)
				{
					web.WebCrawlStatus = CrawlStatus.Error;
					web.UrlCrawlStatus = CrawlStatus.Error;
					web.ErrorMsg = ex.Message;
				}
				Utility.Log("Error:" + web.Url, ex);
				return list;
			}
			finally
			{
				if (resp != null)
					resp.Close();
			}

		}

		private bool IsUseful(string strRef)
		{
			if (WebList.FindIndex(w => w.Url == strRef) > -1)
				return false;

			Uri uri = new Uri(strRef);
			string domain = uri.Host;

			if (WebList.Count(w => w.Domain == domain) > 10)
				return false;

			return true;

		}

		private WebResourceEntity CreateWebResource(string link)
		{
			Uri uri = new Uri(link);
			WebResourceEntity web = new WebResourceEntity();
			web.Url = link;
			web.Domain = uri.Host;
			web.RootDomain = Utility.GetRootDomain(link);
			web.Source = WebSource.Crawler;
			//web.UrlPath = Utility.GetAbsoluteUrl(link);
			return web;
		}
	}
}
